
%pip install --upgrade kneed 
# para que luego funcione el comando : from kneed import KneeLocator

Requirement already satisfied: kneed in c:\users\buba\anaconda3\lib\site-packages (0.8.5)
Requirement already satisfied: scipy>=1.0.0 in c:\users\buba\anaconda3\lib\site-packages (from kneed) (1.9.1)
Requirement already satisfied: numpy>=1.14.2 in c:\users\buba\anaconda3\lib\site-packages (from kneed) (1.21.5)
Note: you may need to restart the kernel to use updated packages.


# cálculos, dataframes, uso genereal
import pandas as pd
import numpy as np

# visualización
import matplotlib.pyplot as plt
# import matplotlib.cm as cmimport matplotlib.cm as cm
import seaborn as sns
import plotly.express as px

from matplotlib.colors import ListedColormap
from matplotlib import colormaps

# list(colormaps)from matplotlib.colors import ListedColormap
from matplotlib import colormaps
# list(colormaps)

from sklearn import preprocessing
from sklearn.preprocessing import OneHotEncoder

# Importamos otras librerías necesarias

from datetime import datetime
from scipy.stats import shapiro

from sklearn.preprocessing import RobustScaler
from sklearn.decomposition import PCA
from sklearn.cluster import KMeans
from kneed import KneeLocator
from scipy.cluster.hierarchy import linkage
from scipy.cluster.hierarchy import dendrogram
from scipy.cluster.hierarchy import fcluster
from scipy.cluster.hierarchy import cophenet
from scipy.spatial.distance import pdist
from sklearn.cluster import AgglomerativeClustering
import sklearn.metrics as sm
from sklearn.metrics import silhouette_samples, silhouette_score

import warnings
warnings.filterwarnings('ignore')


fbthai_data= pd.read_csv('PublicacionsFacebookThailandia.csv')
fbthai_data


fbthai_data.describe().round()


fbthai_data.shape

(7050, 16)


fbthai_data.info()

<class 'pandas.core.frame.DataFrame'>
RangeIndex: 7050 entries, 0 to 7049
Data columns (total 16 columns):
 #   Column            Non-Null Count  Dtype  
---  ------            --------------  -----  
 0   status_id         7050 non-null   object 
 1   status_type       7050 non-null   object 
 2   status_published  7050 non-null   object 
 3   num_reactions     7050 non-null   int64  
 4   num_comments      7050 non-null   int64  
 5   num_shares        7050 non-null   int64  
 6   num_likes         7050 non-null   int64  
 7   num_loves         7050 non-null   int64  
 8   num_wows          7050 non-null   int64  
 9   num_hahas         7050 non-null   int64  
 10  num_sads          7050 non-null   int64  
 11  num_angrys        7050 non-null   int64  
 12  Column1           0 non-null      float64
 13  Column2           0 non-null      float64
 14  Column3           0 non-null      float64
 15  Column4           0 non-null      float64
dtypes: float64(4), int64(9), object(3)
memory usage: 881.4+ KB


fbthai_data.describe().round().T


# Vemos los valores únicos de cada uno de los atributos
fbthai_data.nunique()

status_id           6997
status_type            4
status_published    6913
num_reactions       1067
num_comments         993
num_shares           501
num_likes           1044
num_loves            229
num_wows              65
num_hahas             42
num_sads              24
num_angrys            14
Column1                0
Column2                0
Column3                0
Column4                0
dtype: int64


fbthai_data.isnull().sum() | fbthai_data.isna().sum()

status_id              0
status_type            0
status_published       0
num_reactions          0
num_comments           0
num_shares             0
num_likes              0
num_loves              0
num_wows               0
num_hahas              0
num_sads               0
num_angrys             0
Column1             7050
Column2             7050
Column3             7050
Column4             7050
dtype: int64


# Exploring status_id variable
len(fbthai_data['status_id'].unique())

6997


# Exploring status_published variable
len(fbthai_data['status_published'].unique())

6913


len(fbthai_data['status_type'].unique())
fbthai_data['status_type'].unique()

array(['video', 'photo', 'link', 'status'], dtype=object)


fbthai_data.duplicated().sum()

51


# ver mejor los campos duplicados

duplicated_rows = fbthai_data[fbthai_data.duplicated()]
duplicated_rows


status_id_counts = duplicated_rows['status_id'].value_counts()
print(status_id_counts)

819700534875473_1002372733274918    1
819700534875473_962032993975559     1
819700534875473_966878030157722     1
819700534875473_966629653515893     1
819700534875473_966156923563166     1
819700534875473_965614580284067     1
819700534875473_965407870304738     1
819700534875473_964582740387251     1
819700534875473_963754250470100     1
819700534875473_962799373898921     1
819700534875473_962694150576110     1
819700534875473_962061653972693     1
819700534875473_960466324132226     1
819700534875473_968264653352393     1
819700534875473_960320564146802     1
819700534875473_958291217683070     1
819700534875473_957697307742461     1
819700534875473_957599447752247     1
819700534875473_956769714501887     1
819700534875473_955793427932849     1
819700534875473_955743124604546     1
819700534875473_955149101330615     1
819700534875473_954387151406810     1
819700534875473_953048221540703     1
819700534875473_967787160066809     1
819700534875473_971940472984811     1
819700534875473_1001982519980606    1
819700534875473_991757311003127     1
819700534875473_1000607730118085    1
819700534875473_999880033524188     1
819700534875473_998824716963053     1
819700534875473_997075427137982     1
819700534875473_995933077252217     1
819700534875473_995839410594917     1
819700534875473_995218893990302     1
819700534875473_993975437447981     1
819700534875473_993602007485324     1
819700534875473_993049450873913     1
819700534875473_990066331172225     1
819700534875473_972433889602136     1
819700534875473_988583821320476     1
819700534875473_985229911655867     1
819700534875473_984750728370452     1
819700534875473_983794271799431     1
819700534875473_981835315328660     1
819700534875473_980941638751361     1
819700534875473_978746665637525     1
819700534875473_977201195792072     1
819700534875473_976401089205416     1
819700534875473_974461939399331     1
819700534875473_951614605017398     1
Name: status_id, dtype: int64


# ver el total de los valores únicos en los elementos duplicados 
duplicated_rows.nunique()

status_id           51
status_type          3
status_published    51
num_reactions       49
num_comments        40
num_shares          20
num_likes           51
num_loves           22
num_wows            18
num_hahas            4
num_sads             7
num_angrys           4
Column1              0
Column2              0
Column3              0
Column4              0
dtype: int64


fbthai_data_transform = fbthai_data.drop_duplicates(keep='last')
fbthai_data_transform[fbthai_data_transform.duplicated()]


fbthai_data_transform.info()

<class 'pandas.core.frame.DataFrame'>
Int64Index: 6999 entries, 0 to 7049
Data columns (total 16 columns):
 #   Column            Non-Null Count  Dtype  
---  ------            --------------  -----  
 0   status_id         6999 non-null   object 
 1   status_type       6999 non-null   object 
 2   status_published  6999 non-null   object 
 3   num_reactions     6999 non-null   int64  
 4   num_comments      6999 non-null   int64  
 5   num_shares        6999 non-null   int64  
 6   num_likes         6999 non-null   int64  
 7   num_loves         6999 non-null   int64  
 8   num_wows          6999 non-null   int64  
 9   num_hahas         6999 non-null   int64  
 10  num_sads          6999 non-null   int64  
 11  num_angrys        6999 non-null   int64  
 12  Column1           0 non-null      float64
 13  Column2           0 non-null      float64
 14  Column3           0 non-null      float64
 15  Column4           0 non-null      float64
dtypes: float64(4), int64(9), object(3)
memory usage: 929.6+ KB


# Make a copy of the 'status_id' column
status_id_copy = fbthai_data_transform['status_id'].copy()

# Split the copied column into 'user_id' and 'status_id' columns
fbthai_data_transform[['userID', 'postID']] = status_id_copy.str.split('_', expand=True)


fbthai_data_transform[['status_id']].value_counts()

status_id                       
246675545449582_326883450762124     2
246675545449582_429583263825475     2
246675545449582_935415303242266     1
246675545449582_934896833294113     1
246675545449582_934381930012270     1
                                   ..
246675545449582_1106131959503932    1
246675545449582_1103306073119854    1
246675545449582_1102277329889395    1
246675545449582_1101125083337953    1
848451358648417_978696712290547     1
Length: 6997, dtype: int64


fbthai_data_transform[['userID']].value_counts()

userID          
246675545449582     2635
614855718638584     1440
134115277150304     1212
1050855161656896     778
848451358648417      450
725980084255106      252
146481362756646      125
117059915488335       56
819700534875473       51
dtype: int64


import matplotlib.pyplot as plt

# Count the occurrences of each user ID
user_id_counts = fbthai_data_transform['userID'].value_counts()

# Plot the bar plot
user_id_counts.plot(kind='bar', figsize=(10, 6))
plt.title('Count of Values for Each User ID')
plt.xlabel('User ID')
plt.ylabel('Count')
plt.show()


import matplotlib.pyplot as plt

# Group the data by 'userID' and 'status_type', and count the occurrences
user_status_counts = fbthai_data_transform.groupby('userID')['status_type'].value_counts()

# Get unique user IDs and their counts
unique_user_ids = fbthai_data_transform['userID'].unique()
user_counts = [user_status_counts[user_id].sum() for user_id in unique_user_ids]

# Sort user IDs based on counts
sorted_user_ids = [user_id for _, user_id in sorted(zip(user_counts, unique_user_ids), reverse=True)]

# Calculate number of rows and columns needed
num_rows = 3
num_cols = 3
num_plots = min(len(sorted_user_ids), num_rows * num_cols)

# Plot for each user ID
fig, axes = plt.subplots(num_rows, num_cols, figsize=(15, 10))

for i, user_id in enumerate(sorted_user_ids[:num_plots]):
    row_index = i // num_cols
    col_index = i % num_cols
    user_data = user_status_counts[user_id]
    user_data.plot(kind='bar', ax=axes[row_index, col_index])
    axes[row_index, col_index].set_title(f'User ID {user_id}')
    axes[row_index, col_index].set_xlabel('Status Type')
    axes[row_index, col_index].set_ylabel('Count')
    axes[row_index, col_index].tick_params(axis='x', rotation=45)

# Hide empty subplots if there are fewer plots than total number of subplots
for i in range(num_plots, num_rows * num_cols):
    row_index = i // num_cols
    col_index = i % num_cols
    axes[row_index, col_index].axis('off')

plt.tight_layout()
plt.show()


import matplotlib.pyplot as plt

# Count the occurrences of each user ID
user_id_counts = fbthai_data_transform['status_type'].value_counts()

# Plot the bar plot
user_id_counts.plot(kind='bar', figsize=(10, 6))
plt.title('Total status types comprising all Users IDs')
plt.xlabel('Media type')
plt.ylabel('Count')
plt.show()


fbthai_data_transform.info()

<class 'pandas.core.frame.DataFrame'>
Int64Index: 6999 entries, 0 to 7049
Data columns (total 18 columns):
 #   Column            Non-Null Count  Dtype  
---  ------            --------------  -----  
 0   status_id         6999 non-null   object 
 1   status_type       6999 non-null   object 
 2   status_published  6999 non-null   object 
 3   num_reactions     6999 non-null   int64  
 4   num_comments      6999 non-null   int64  
 5   num_shares        6999 non-null   int64  
 6   num_likes         6999 non-null   int64  
 7   num_loves         6999 non-null   int64  
 8   num_wows          6999 non-null   int64  
 9   num_hahas         6999 non-null   int64  
 10  num_sads          6999 non-null   int64  
 11  num_angrys        6999 non-null   int64  
 12  Column1           0 non-null      float64
 13  Column2           0 non-null      float64
 14  Column3           0 non-null      float64
 15  Column4           0 non-null      float64
 16  userID            6999 non-null   object 
 17  postID            6999 non-null   object 
dtypes: float64(4), int64(9), object(5)
memory usage: 1.0+ MB


# separo la columna con el espacio

fbthai_data_transform[['Column3', 'Column4']] = fbthai_data_transform['status_published'].str.split(' ', expand=True)

fbthai_data_transform = fbthai_data_transform.rename(columns={'Column3': 'day', 'Column4': 'hour'})#canvio el nom a la columna
fbthai_data_transform['day'] = pd.to_datetime(fbthai_data_transform['day']) #Poso format de dia a la columna day

from datetime import datetime #Poso format hora a la columna hour
fbthai_data_transform['hour'] = fbthai_data_transform['hour'].apply(lambda x: datetime.strptime(x, '%H:%M').time())


# Dummy encode 'user_id' column
dummies = pd.get_dummies(fbthai_data_transform['userID'], prefix='ID', drop_first=False).astype('int64')
dummies


# Concatenate the dummy columns with the original DataFrame
fbthai_data_transform_dummy = pd.concat([fbthai_data_transform, dummies], axis=1)


fbthai_data_transform_dummy.info()

<class 'pandas.core.frame.DataFrame'>
Int64Index: 6999 entries, 0 to 7049
Data columns (total 27 columns):
 #   Column               Non-Null Count  Dtype         
---  ------               --------------  -----         
 0   status_id            6999 non-null   object        
 1   status_type          6999 non-null   object        
 2   status_published     6999 non-null   object        
 3   num_reactions        6999 non-null   int64         
 4   num_comments         6999 non-null   int64         
 5   num_shares           6999 non-null   int64         
 6   num_likes            6999 non-null   int64         
 7   num_loves            6999 non-null   int64         
 8   num_wows             6999 non-null   int64         
 9   num_hahas            6999 non-null   int64         
 10  num_sads             6999 non-null   int64         
 11  num_angrys           6999 non-null   int64         
 12  Column1              0 non-null      float64       
 13  Column2              0 non-null      float64       
 14  day                  6999 non-null   datetime64[ns]
 15  hour                 6999 non-null   object        
 16  userID               6999 non-null   object        
 17  postID               6999 non-null   object        
 18  ID_1050855161656896  6999 non-null   int64         
 19  ID_117059915488335   6999 non-null   int64         
 20  ID_134115277150304   6999 non-null   int64         
 21  ID_146481362756646   6999 non-null   int64         
 22  ID_246675545449582   6999 non-null   int64         
 23  ID_614855718638584   6999 non-null   int64         
 24  ID_725980084255106   6999 non-null   int64         
 25  ID_819700534875473   6999 non-null   int64         
 26  ID_848451358648417   6999 non-null   int64         
dtypes: datetime64[ns](1), float64(2), int64(18), object(6)
memory usage: 1.5+ MB


fbthai_data_transform_dummy['Saturday'] = fbthai_data_transform_dummy['day'].dt.weekday.isin([5]).astype(int)
fbthai_data_transform_dummy['Sunday'] = fbthai_data_transform_dummy['day'].dt.weekday.isin([6]).astype(int)
fbthai_data_transform_dummy['Workday'] = (fbthai_data_transform_dummy['day'].dt.weekday < 5).astype(int)


# separamos las hores en periodos de publicación

def time_of_day(hour):
    if hour.hour >= 6 and hour.hour < 12:
        return 'morning'
    elif hour.hour >= 12 and hour.hour < 22:
        return 'afternoon/evening'
    else:
        return 'night'

fbthai_data_transform_dummy['time_day'] = fbthai_data_transform_dummy['hour'].apply(time_of_day)

# extrayendo los años
fbthai_data_transform_dummy['year'] = fbthai_data_transform_dummy['day'].dt.year
fbthai_data_transform_dummy.head()


# Agroupar data por status_type + calcular la suma de las interacciones medidas

data_status_year = fbthai_data_transform_dummy.groupby(['status_type', 'year'])[['num_reactions', 'num_comments', 'num_shares', 'num_likes']].sum()
data_status_year = data_status_year.reset_index()


fig, ax = plt.subplots(figsize=(12, 5))
sns.barplot(data=data_status_year, x='year', y='num_reactions', hue='status_type', ax=ax)
ax.set_xlabel('Years')
ax.set_ylabel('Total Reactions')
ax.set_title('Interaction Totals by Status Type and Year')
plt.legend(title='Media Type')
plt.show()


# Sum the counts of each status type for each year
data_status_year_cumulated = data_status_year.groupby('year')['num_reactions', 'num_comments', 'num_shares', 'num_likes'].sum().reset_index()

# Melt the DataFrame to have 'status_type' as a variable
data_status_year_cumulated_melted = data_status_year_cumulated.melt(id_vars='year', var_name='interaction_type', value_name='total_count')

# Plot the total number of status types cumulated for each year
fig, ax = plt.subplots(figsize=(12, 5))
sns.barplot(data=data_status_year_cumulated_melted, x='year', y='total_count', hue='interaction_type', ax=ax)
ax.set_xlabel('Years')
ax.set_ylabel('Total Count')
ax.set_title('Cumulated Total Status Types by Year')
plt.legend(title='Interaction Type')
plt.show()


# Group by 'time_day' and 'status_type' and sum the counts of interactions
data_status_timeday = fbthai_data_transform_dummy.groupby(['time_day', 'status_type'])[['num_reactions', 'num_comments', 'num_shares', 'num_likes', 'num_loves']].sum().reset_index()

# Set up the figure and axes
fig, ax = plt.subplots(figsize=(12, 5))

# Generate the grouped bar plot
sns.barplot(data=data_status_timeday, x='time_day', y='num_reactions', hue='status_type', ax=ax)
ax.set_ylabel('Total Count')
ax.set_title('Publications fron sellers by Status Type and media used by time of the dat')
plt.legend(title='Status Type')

plt.show()


data_status_timeday = fbthai_data_transform_dummy.groupby('time_day')[['num_reactions', 'num_comments', 'num_shares', 'num_likes','num_loves']].sum()
data_status_timeday = data_status_timeday.reset_index()

# Set up the figure and axes
fig, ax = plt.subplots(figsize=(12, 5))

# Generate the grouped bar plot

data_status_timeday.plot(kind='bar', x='time_day', y=['num_reactions', 'num_comments', 'num_shares', 'num_likes', 'num_loves'], ax=ax)
ax.set_ylabel('Total Count')
ax.set_title('Audence interactions by Time of the day')
plt.legend(title='Interaction Type')

plt.show()


# Datos para el gráfico
dataday = fbthai_data_transform_dummy.groupby(['Saturday', 'Sunday', 'Workday'])[['num_reactions', 'num_comments', 'num_shares', 'num_likes', 'num_loves']].sum()

# Create a single plot
fig, ax = plt.subplots(figsize=(12, 4))

# Plot the data
dataday.plot(kind='bar', ax=ax)
ax.set_xticklabels(['Saturday', 'Sunday', 'Workday'], rotation=0)
ax.set_xlabel('Days of the Week')
ax.set_ylabel('Total')
ax.set_title('Interaction Totals by Day of the Week')
ax.legend(title='Metrics by Day of the Week')

plt.show()


fbthai_data_transform_dummy


# Get dummies for 'status_type' column
status_dummies = pd.get_dummies(fbthai_data_transform_dummy['status_type'], prefix='status_type', drop_first=False)

# Get dummies for 'time_day' column
time_dummies = pd.get_dummies(fbthai_data_transform_dummy['time_day'], prefix='time_day', drop_first=False)

# Concatenate the dummy variables
dummies = pd.concat([status_dummies, time_dummies], axis=1)

# Convert dummy columns to numeric values (0 and 1)
dummies = dummies.astype(int)

# Afegim les noves variables dummies al df original
df = pd.concat([fbthai_data_transform_dummy, dummies], axis=1)


# Convertim 'status_published' a tipus datetime
df['status_published'] = pd.to_datetime(df['status_published'])

# Creem noves columnes numèriques
df['month'] = df['status_published'].dt.month
df['day'] = df['status_published'].dt.day
df['hour'] = df['status_published'].dt.hour
df['minute'] = df['status_published'].dt.minute


df.head()


df.drop(['Column1', 'Column2', 'status_id', 'status_published', 'status_type', 'userID', 'postID', 'time_day'], axis=1, inplace=True)


# Extract column names starting with "ID_"
id_columns = [col for col in df.columns if col.startswith("ID_")]

# Extract non-"ID_" columns
non_id_columns = [col for col in df.columns if not col.startswith("ID_")]

# Reorder columns with non-"ID_" columns first and "ID_" columns last
new_order = non_id_columns + id_columns

# Create new DataFrame with reordered columns
df = df[new_order]
df


df.info()

<class 'pandas.core.frame.DataFrame'>
Int64Index: 6999 entries, 0 to 7049
Data columns (total 33 columns):
 #   Column                      Non-Null Count  Dtype
---  ------                      --------------  -----
 0   num_reactions               6999 non-null   int64
 1   num_comments                6999 non-null   int64
 2   num_shares                  6999 non-null   int64
 3   num_likes                   6999 non-null   int64
 4   num_loves                   6999 non-null   int64
 5   num_wows                    6999 non-null   int64
 6   num_hahas                   6999 non-null   int64
 7   num_sads                    6999 non-null   int64
 8   num_angrys                  6999 non-null   int64
 9   day                         6999 non-null   int64
 10  hour                        6999 non-null   int64
 11  Saturday                    6999 non-null   int32
 12  Sunday                      6999 non-null   int32
 13  Workday                     6999 non-null   int32
 14  year                        6999 non-null   int64
 15  status_type_link            6999 non-null   int32
 16  status_type_photo           6999 non-null   int32
 17  status_type_status          6999 non-null   int32
 18  status_type_video           6999 non-null   int32
 19  time_day_afternoon/evening  6999 non-null   int32
 20  time_day_morning            6999 non-null   int32
 21  time_day_night              6999 non-null   int32
 22  month                       6999 non-null   int64
 23  minute                      6999 non-null   int64
 24  ID_1050855161656896         6999 non-null   int64
 25  ID_117059915488335          6999 non-null   int64
 26  ID_134115277150304          6999 non-null   int64
 27  ID_146481362756646          6999 non-null   int64
 28  ID_246675545449582          6999 non-null   int64
 29  ID_614855718638584          6999 non-null   int64
 30  ID_725980084255106          6999 non-null   int64
 31  ID_819700534875473          6999 non-null   int64
 32  ID_848451358648417          6999 non-null   int64
dtypes: int32(10), int64(23)
memory usage: 1.5 MB


df.describe().round().T


columnes_numeriques = df.select_dtypes(include=['number']).columns
columnes_numeriques

Index(['num_reactions', 'num_comments', 'num_shares', 'num_likes', 'num_loves',
       'num_wows', 'num_hahas', 'num_sads', 'num_angrys', 'day', 'hour',
       'Saturday', 'Sunday', 'Workday', 'year', 'status_type_link',
       'status_type_photo', 'status_type_status', 'status_type_video',
       'time_day_afternoon/evening', 'time_day_morning', 'time_day_night',
       'month', 'minute', 'ID_1050855161656896', 'ID_117059915488335',
       'ID_134115277150304', 'ID_146481362756646', 'ID_246675545449582',
       'ID_614855718638584', 'ID_725980084255106', 'ID_819700534875473',
       'ID_848451358648417'],
      dtype='object')


import seaborn as sns
import matplotlib.pyplot as plt

# Calculate the correlation matrix
corr = df.corr()

# Generate a mask for the upper triangle
mask = np.triu(np.ones_like(corr, dtype=bool))

# Set up the matplotlib figure
fig, ax = plt.subplots(figsize=(25, 25))

# Generate a custom diverging colormap
cmap = sns.diverging_palette(230, 20, as_cmap=True)

# Draw the heatmap with the mask and correct aspect ratio
sns.heatmap(corr, mask=mask, cmap=cmap, vmax=1, center=0,
            square=True, linewidths=.5, cbar_kws={"shrink": .5}, annot=True, fmt='.1f')

plt.show()


fig, ax = plt.subplots(figsize=(16, 6))
ax = sns.boxplot(data = df, linewidth= 0.5)
plt.title('columnes_numeriques : distribución ', fontsize=14)
ax.set_xticklabels(ax.get_xticklabels(), rotation=75) 
plt.show()


num_columns = df[['num_reactions', 'num_comments', 'num_shares', 'num_likes', 'num_loves',
                  'num_wows', 'num_hahas', 'num_sads', 'num_angrys', 'year', 'month',
                  'day', 'hour', 'minute']]

fig, ax = plt.subplots(figsize=(14, 6))
ax = sns.boxplot(data=num_columns, linewidth=0.5)
ax.set_xticklabels(ax.get_xticklabels(), rotation=75) 
plt.title('columnes_numeriques: distribución', fontsize=14)
plt.show()


# Comprobamos mediante shapiro qué variables presentan distribución gaussinana

from scipy.stats import shapiro
import colorama
from colorama import Fore, Style

col = list(df.columns)

def distr_normal(df):
    normal = []
    for col in df.columns:
        p_valor = shapiro(df[col])[1]
        normal.append(p_valor > 0.05)
    return normal
resultado = distr_normal(df)

for resultado in resultado :
    if resultado is True : 
        print(Fore.GREEN, resultado, Style.RESET_ALL)
    else :
        print( Fore.RED, resultado, Style.RESET_ALL)

 False 
 False 
 False 
 False 
 False 
 False 
 False 
 False 
 False 
 False 
 False 
 False 
 False 
 False 
 False 
 False 
 False 
 False 
 False 
 False 
 False 
 False 
 False 
 False 
 False 
 False 
 False 
 False 
 False 
 False 
 False 
 False 
 False


import matplotlib.pyplot as plt
import seaborn as sns
import pandas as pd

# Seleccionem només les columnes numèriques
columnes_numeriques = df.select_dtypes(include=['number']).columns

# Determinem el nombre de files i columnes per als subplots
n_cols = 6  # Increase the number of columns
n_rows = (len(columnes_numeriques) + n_cols - 1) // n_cols

# Creem els subplots amb mides ajustades
fig, axs = plt.subplots(nrows=n_rows, ncols=n_cols, figsize=(14, 3 * n_rows))  # Adjust figsize for larger subplots

# Aplanem la matriu d'axis per iterar fàcilment
axs = axs.flatten()

# Creem un boxplot per a cada columna numèrica
for i, col in enumerate(columnes_numeriques):
    sns.boxplot(y=df[col], ax=axs[i], orient='v')  # Set orient='v' for vertical orientation
    axs[i].set_title(col)

# Amaguem els gràfics no utilitzats si n'hi ha
for i in range(len(columnes_numeriques), len(axs)):
    axs[i].set_visible(False)

# Ajustem el layout per evitar la superposició
plt.tight_layout()

# Mostrem el gràfic
plt.show()


# Calculate outliers for each column and count them
outliers_count = {}

for col in columnes_numeriques:
    q1 = df[col].quantile(0.25)
    q3 = df[col].quantile(0.75)
    iqr = q3 - q1
    lower_bound = q1 - 1.5 * iqr
    upper_bound = q3 + 1.5 * iqr
    
    outliers = df[(df[col] < lower_bound) | (df[col] > upper_bound)][col]
    outliers_count[col] = len(outliers)

    
# Print the number of outliers for each variable
print("\nNumber of outliers for each variable:")
for col, count in outliers_count.items():
    if count == 0:
        print(f"\033[92m{col}: {count}\033[0m")  # Print in green
    else:
        print(f"\033[91m{col}: {count}\033[0m")  # Print in red

Number of outliers for each variable:
num_reactions: 689
num_comments: 1365
num_shares: 1382
num_likes: 754
num_loves: 1303
num_wows: 1702
num_hahas: 1117
num_sads: 598
num_angrys: 420
day: 0
hour: 1068
Saturday: 976
Sunday: 1050
Workday: 0
year: 589
status_type_link: 63
status_type_photo: 0
status_type_status: 359
status_type_video: 0
time_day_afternoon/evening: 408
time_day_morning: 0
time_day_night: 0
month: 0
minute: 0
ID_1050855161656896: 778
ID_117059915488335: 56
ID_134115277150304: 1212
ID_146481362756646: 125
ID_246675545449582: 0
ID_614855718638584: 1440
ID_725980084255106: 252
ID_819700534875473: 51
ID_848451358648417: 450


import matplotlib.pyplot as plt
import seaborn as sns
import pandas as pd

# Select only the numerical columns
numerical_columns = df.select_dtypes(include=['number']).columns

# Determine the number of rows and columns for subplots
n_cols = 4  # Increase the number of columns
n_rows = (len(numerical_columns) + n_cols - 1) // n_cols

# Create subplots with adjusted sizes
fig, axs = plt.subplots(nrows=n_rows, ncols=n_cols, figsize=(15, 4 * n_rows))  # Adjust figsize for larger subplots

# Flatten the axis matrix for easy iteration
axs = axs.flatten()

# Create a histogram for each numerical column
for i, col in enumerate(numerical_columns):
    sns.histplot(df[col], ax=axs[i], kde=True)  # Plot histogram with kernel density estimation
    axs[i].set_title(col)

# Hide unused plots if any
for i in range(len(numerical_columns), len(axs)):
    axs[i].set_visible(False)

# Adjust layout to avoid overlap
plt.tight_layout()

# Show the plot
plt.show()


from sklearn.preprocessing import MinMaxScaler, RobustScaler
import pandas as pd

# Creamos una copia del DataFrame para evitar modificar el original
df_processed = df.copy()

# Escalado de variables numéricas sin outliers usando MinMaxScaler
numeric_columns_minmax = ['month', 'day', 'minute']
scaler_minmax = MinMaxScaler()
df_processed[numeric_columns_minmax] = scaler_minmax.fit_transform(df_processed[numeric_columns_minmax])

# Escalado de variables numéricas con outliers usando RobustScaler
numeric_columns_robust = ['num_reactions', 'num_comments', 'num_shares', 'num_likes', 'num_loves',
                          'num_wows', 'num_hahas', 'num_sads', 'num_angrys', 'year', 'hour']
scaler_robust = RobustScaler()
df_processed[numeric_columns_robust] = scaler_robust.fit_transform(df_processed[numeric_columns_robust])

df_processed.head()


fig, ax = plt.subplots(figsize=(15, 8))
ax = sns.boxplot(data = df_processed[['num_reactions', 'num_comments', 'num_shares', 'num_likes', 'num_loves',
                  'num_wows', 'num_hahas', 'num_sads', 'num_angrys', 'year', 'month',
                  'day', 'hour', 'minute']], linewidth= 0.5)
ax.set_xticklabels(ax.get_xticklabels(), rotation=75) 
plt.title('El dataset df_processed escalado, viendo solo las variables estríctamente numéricas', fontsize=14)
plt.show()


from sklearn.cluster import KMeans
from sklearn.metrics import silhouette_score
import matplotlib.pyplot as plt
from kneed import KneeLocator

# Creamos listas para almacenar los valores de WCSS y las puntuaciones de la silhouette
wcss = []
silhouette_coefficients = []

# Decidimos el rango de k para probar
range_k = range(2, 11)

# Iteramos sobre el rango de valores de K
for k in range_k:
    # Instanciamos el modelo KMeans con el número deseado de clusters
    kmeans = KMeans(n_clusters=k, n_init=10, random_state=42)
    
    # Ajustamos el modelo a los datos
    kmeans.fit(df_processed)
    
    # Guardamos el resultado del ajuste para posibles usos posteriores
    cluster_labels = kmeans.labels_
    
    # Calculamos la suma de los cuadrados intra-cluster (WCSS) y la agregamos a la lista
    wcss_value = kmeans.inertia_
    wcss.append(wcss_value)
    print(f"Valor de WCSS para k={k}: {wcss_value}")
    
    # Calculamos la puntuación de la silhouette y la agregamos a la lista
    silhouette_score_value = silhouette_score(df_processed, cluster_labels)
    silhouette_coefficients.append(silhouette_score_value)
    print(f"Silhouette coefficients para k={k}: {silhouette_score_value}", '\n')

# Resto del código para crear Elbow Plot y mostrar las puntuaciones de la silhouette...


# Creamos una instancia de KneeLocator para encontrar el punto óptimo (codo)
knee = KneeLocator(range(2, 11), wcss, curve='convex', direction='decreasing')

# Encontramos el punto óptimo (codo) utilizando el método elbow
optimal_k = knee.elbow

# Graficamos el Elbow Plot con el punto óptimo destacado
plt.figure(figsize=(10, 6))
plt.plot(range_k, wcss, 'bx-', label='WCSS')
plt.xlabel('k')
plt.ylabel('WCSS')
plt.title('Elbow Plot y silhouette para determinar el valor óptimo de k')
plt.xticks(range(2, 11))
plt.grid(True)
plt.axvline(x=optimal_k, linestyle='--', color='r', label='Punto óptimo (codo)')
plt.legend(loc='upper right')

# Graficamos las puntuaciones de la silhouette
plt.twinx()
plt.plot(range_k, silhouette_coefficients, 'go-', label='silhouette')
plt.ylabel('Silhouette coefficients')
plt.legend(loc='lower right')

plt.show()

Valor de WCSS para k=2: 10027095.86832271
Silhouette coefficients para k=2: 0.8853714287807128 

Valor de WCSS para k=3: 6988957.191347623
Silhouette coefficients para k=3: 0.8383762670013402 

Valor de WCSS para k=4: 5533836.271135641
Silhouette coefficients para k=4: 0.8124180710128246 

Valor de WCSS para k=5: 4387030.243782711
Silhouette coefficients para k=5: 0.8185708672002341 

Valor de WCSS para k=6: 3659940.5363405785
Silhouette coefficients para k=6: 0.8126576595822413 

Valor de WCSS para k=7: 3008888.556921824
Silhouette coefficients para k=7: 0.8127290297105586 

Valor de WCSS para k=8: 2622722.7478284203
Silhouette coefficients para k=8: 0.7994551569624816 

Valor de WCSS para k=9: 2372309.2135463273
Silhouette coefficients para k=9: 0.8024257604082419 

Valor de WCSS para k=10: 2190874.297948455
Silhouette coefficients para k=10: 0.7879934031222525


# Creamos una instancia del modelo KMeans con el número deseado de clusters
kmeans = KMeans(n_clusters=5, random_state=42)

# Ajustamos el modelo a los datos
kmeans.fit(df_processed)

# Obtenemos las etiquetas de cluster asignadas a cada registro
cluster_labels = kmeans.labels_


# Agregar las etiquetas de cluster al DataFrame original
df_processed['cluster_label'] = cluster_labels

# Agrupar los datos por las etiquetas de cluster y mostrar el contenido de cada grupo
cluster_groups = df_processed.groupby('cluster_label')


df_processed.head()


# Agregar las etiquetas de cluster al DataFrame original
df_processed['cluster_label'] = cluster_labels

# Contar el número de registros en cada cluster
cluster_counts = df_processed['cluster_label'].value_counts()

# Mostrar el número de registros en cada cluster
print("Número de registros en cada cluster:")
print(cluster_counts)

Número de registros en cada cluster:
0    6163
4     605
1     113
2      85
3      33
Name: cluster_label, dtype: int64


# Agregar las etiquetas de cluster al DataFrame original
df_processed['cluster_label'] = cluster_labels

# Contar el número de registros en cada cluster
cluster_counts = df_processed['cluster_label'].value_counts()

# Calcular porcentaje de registros en cada cluster
percentages = cluster_counts / cluster_counts.sum() * 100

# Set up the figure with two subplots
fig, axs = plt.subplots(1, 2, figsize=(14, 6))

# Pie chart
patches, texts = axs[0].pie(cluster_counts, labels=None, colors=sns.color_palette('Set1'))
axs[0].set_title('Porcentaje de registros en cada cluster')

# Construir la leyenda con los valores numéricos y los porcentajes
legend_labels = [f'Cluster {idx}: {value} ({percent:.1f}%)' for idx, (value, percent) in enumerate(zip(cluster_counts, percentages))]

# Add legend outside the plot
axs[0].legend(patches, legend_labels, loc='center left', bbox_to_anchor=(1, 0.5))

# Bar chart
cluster_counts.plot(kind='bar', ax=axs[1], color=sns.color_palette('Set1'))
axs[1].set_title('Número de registros en cada cluster')
axs[1].set_xlabel('Cluster')
axs[1].set_ylabel('Número de registros')
axs[1].grid(axis='y', linestyle='--', alpha=0.7)

# Adjust layout
plt.tight_layout()

plt.show()


import matplotlib.pyplot as plt
import seaborn as sns

# Function to create scatter plots
def create_scatterplot(x, y, data, hue, palette, centroids):
    ax = sns.scatterplot(x=x, y=y, data=data, hue=hue, palette=palette, alpha = 0.8)
    ax.scatter(centroids[:, 0], centroids[:, 1], marker='+', c='red', s=100, label='Centroids')
    ax.set_xlabel(x)
    ax.set_ylabel(y)

# Define color palette
palette = 'viridis'

# Tamaño de la figura
plt.figure(figsize=(14, 8))

# Subgráfico 1
plt.subplot(2, 3, 1)
create_scatterplot('num_shares', 'num_comments', df_processed, 'cluster_label', palette, centroids)
plt.title('num_shares vs num_comments')

# Subgráfico 2
plt.subplot(2, 3, 2)
create_scatterplot('num_likes', 'num_reactions', df_processed, 'cluster_label', palette, centroids)
plt.title('num_reactions vs num_likes')

# Subgráfico 3
plt.subplot(2, 3, 3)
create_scatterplot('num_likes', 'num_loves', df_processed, 'cluster_label', palette, centroids)
plt.title('num_likes vs num_loves')

# Subgráfico 4 (el cuarto gráfico que deseas agregar)
plt.subplot(2, 3, 4)
create_scatterplot('num_loves', 'num_shares', df_processed, 'cluster_label', palette, centroids)
plt.title('num_loves vs num_shares')

plt.subplot(2, 3, 5)
create_scatterplot('num_reactions', 'num_comments', df_processed, 'cluster_label', palette, centroids)
plt.title('num_reactions vs num_comments')

plt.tight_layout()  # Ajusta automáticamente los subgráficos para evitar superposiciones
plt.show()


import matplotlib.pyplot as plt

# Filter the DataFrame by cluster_label and sum the values for each cluster
cluster_sum = df_processed.groupby('cluster_label').sum().astype(int)

# Transpose the DataFrame for easier plotting
cluster_sum = cluster_sum.T

# Define colors for each variable
colors = ['skyblue', 'salmon', 'lightgreen', 'orange', 'purple', 'yellow', 'cyan', 'magenta', 'lime', 'brown', 'teal', 'lavender', 'olive', 'pink']

# Create bar graphs for each cluster
for cluster in cluster_sum.columns:
    plt.figure(figsize=(14, 6))
    plt.bar(cluster_sum.index, cluster_sum[cluster], color=colors)
    plt.title(f'Cluster {cluster} - Sum of Values')
    plt.xlabel('Variables')
    plt.ylabel('Sum')
    plt.xticks(rotation=90)
    plt.grid(axis='y', linestyle='--', alpha=0.7)
    plt.tight_layout()
    plt.show()


from sklearn.decomposition import PCA
import matplotlib.pyplot as plt

# Seleccionar las columnas que han sido escaladas
'''scaled_columns = ['num_reactions', 'num_comments', 'num_shares', 'num_likes', 'num_loves',
                  'num_wows', 'num_hahas', 'num_sads', 'num_angrys', 'year', 'month',
                  'day', 'hour', 'minute']
'''

# Seleccionar las columnas que han sido escaladas
scaled_columns = numeric_columns_minmax + numeric_columns_robust 

# Aplicar PCA a las columnas escaladas
pca = PCA()
df_pca = pca.fit_transform(df_processed[scaled_columns])

# Obtener la varianza explicada por cada componente principal
explained_variance_ratio = pca.explained_variance_ratio_


# Imprimir la varianza explicada por cada componente principal
for i, var_ratio in enumerate(explained_variance_ratio):
    print(f"Varianza explicada por el componente principal {i + 1}: {var_ratio:.2f}")

# Calcular la varianza explicada acumulada
cumulative_variance_ratio = np.cumsum(explained_variance_ratio)

# Trazar el gráfico del codo
plt.figure(figsize=(10, 6))
plt.plot(range(1, len(cumulative_variance_ratio) + 1), cumulative_variance_ratio, marker='o', linestyle='-')
plt.title('Número de Componentes Principales PCA')
plt.xlabel('Número de Componentes Principales')
plt.ylabel('Varianza Explicada Acumulada')
plt.grid(True)
plt.show()

# Plot the first two principal components
plt.figure(figsize=(10, 8))
plt.scatter(df_pca[:, 0], df_pca[:, 1], alpha=0.7)
plt.title('PCA Scatter Plot of the First Two Components')
plt.xlabel('Principal Component 1')
plt.ylabel('Principal Component 2')
plt.grid(True)
plt.show()

Varianza explicada por el componente principal 1: 0.78
Varianza explicada por el componente principal 2: 0.17
Varianza explicada por el componente principal 3: 0.02
Varianza explicada por el componente principal 4: 0.01
Varianza explicada por el componente principal 5: 0.00
Varianza explicada por el componente principal 6: 0.00
Varianza explicada por el componente principal 7: 0.00
Varianza explicada por el componente principal 8: 0.00
Varianza explicada por el componente principal 9: 0.00
Varianza explicada por el componente principal 10: 0.00
Varianza explicada por el componente principal 11: 0.00
Varianza explicada por el componente principal 12: 0.00
Varianza explicada por el componente principal 13: 0.00
Varianza explicada por el componente principal 14: 0.00


# Transform the data into the principal component space
pca_result = pca.transform(df_processed[scaled_columns])

# Extract only the first two principal components
pca1 = pca_result[:, 0]
pca2 = pca_result[:, 1]

# Create a DataFrame with the two principal components
df_pca = pd.DataFrame({'pca1': pca1, 'pca2': pca2})
df_pca


# Creamos un dendrograma para visualizar la estructura jerárquica de los clusters
plt.figure(figsize=(14, 10))
dendrogram = sch.dendrogram(sch.linkage(df_pca, method='ward'), color_threshold=200)  # Establecemos el color_threshold adecuado
plt.title('Dendrograma')
plt.xlabel('Registros')
plt.ylabel('Distancia Euclidiana')

# Agregar una línea de umbral para muchos clusters
plt.axhline(y=200, color='r', linestyle='--', label='Umbral para más de 10 clusters')

plt.gca().patch.set_facecolor('white')  # Establecer el fondo blanco
plt.grid(False)  # Quitar cualquier cuadrícula
plt.legend()  # Mostrar la leyenda
plt.show()


from scipy.cluster.hierarchy import fcluster

# Calcula los clusters
clusters = fcluster(sch.linkage(df_pca, method='ward'), 5, criterion='maxclust')

# Calcula el umbral para 5 clusters
threshold = sch.linkage(df_pca, method='ward')[-(5-1), 2]

# Creamos un dendrograma para visualizar la estructura jerárquica de los clusters
plt.figure(figsize=(16, 10))
dendrogram = sch.dendrogram(sch.linkage(df_pca, method='ward'), color_threshold=threshold)
plt.title('Dendrograma')
plt.xlabel('Registros')
plt.ylabel('Distancia Euclidiana')

# Agregar una línea de umbral para 5 clusters
plt.axhline(y=threshold, color='r', linestyle='--', label='Umbral para 5 clusters')

plt.gca().patch.set_facecolor('white')  # Establecer el fondo blanco
plt.grid(False)  # Quitar cualquier cuadrícula
plt.legend()  # Mostrar la leyenda
plt.show()


from sklearn.cluster import KMeans, AgglomerativeClustering
from sklearn.metrics import silhouette_score
from sklearn.decomposition import PCA

# KMeans before PCA
kmeans_before_pca = KMeans(n_clusters=5, random_state=42)
kmeans_labels_before_pca = kmeans_before_pca.fit_predict(df_processed)

# Calculate silhouette score for KMeans before PCA
silhouette_score_before_pca = silhouette_score(df_processed, kmeans_labels_before_pca)
print("Silhouette Score for KMeans before PCA:", silhouette_score_before_pca)
#-------------------------------------------------------------------------------
# Perform PCA
pca = PCA()
df_pca = pca.fit_transform(df_processed[scaled_columns])

# KMeans after PCA
kmeans_after_pca = KMeans(n_clusters=5, random_state=42)
kmeans_labels_after_pca = kmeans_after_pca.fit_predict(df_pca)

# Calculate silhouette score for KMeans after PCA
silhouette_score_after_pca = silhouette_score(df_pca, kmeans_labels_after_pca)
print("Silhouette Score for KMeans after PCA:", silhouette_score_after_pca)
#-------------------------------------------------------------------------------
# Agglomerative Clustering
hierarchical_clustering = AgglomerativeClustering(n_clusters=5)
hierarchical_labels = hierarchical_clustering.fit_predict(df_pca)

# Calculate silhouette score for Agglomerative Clustering
silhouette_score_agglomerative = silhouette_score(df_pca, hierarchical_labels)
print("Silhouette Score for Agglomerative Clustering:", silhouette_score_agglomerative)

Silhouette Score for KMeans before PCA: 0.8189231257536157
Silhouette Score for KMeans after PCA: 0.8257963658284507
Silhouette Score for Agglomerative Clustering: 0.8070151412870082


import matplotlib.pyplot as plt
from sklearn.cluster import AgglomerativeClustering
import scipy.cluster.hierarchy as sch

# Creamos una instancia del modelo AgglomerativeClustering
# Probamos diferentes números de clústeres
min_clusters = 2
max_clusters = 10
scores = []

for n_clusters in range(min_clusters, max_clusters + 1):
    hierarchical_clustering = AgglomerativeClustering(n_clusters=n_clusters)
    hierarchical_labels = hierarchical_clustering.fit_predict(df_pca)
    score = silhouette_score(df_pca, hierarchical_labels)
    scores.append(score)

# Encontramos el número óptimo de clústeres utilizando el método de la silueta
optimal_n_clusters = min_clusters + scores.index(max(scores))
print('\n'"Número óptimo de clústeres (utilizando Silhouette):", optimal_n_clusters)

# Creamos un dendrograma para visualizar la estructura jerárquica de los clústeres
plt.figure(figsize=(14, 10))
dendrogram = sch.dendrogram(sch.linkage(df_pca, method='ward'), labels=hierarchical_labels)

# Agregar una línea de umbral para n clusters
plt.axhline(y=threshold, color='r', linestyle='--', label='Umbral para n clusters')

plt.title('Dendrograma con Nombres de Clústeres')
plt.xlabel('Registros')
plt.ylabel('Distancia Euclidiana')
plt.gca().patch.set_facecolor('white')  # Establecer el fondo blanco
plt.grid(False)  # Quitar cualquier cuadrícula
plt.show()

Número óptimo de clústeres (utilizando Silhouette): 2


# Imprimir la lista de scores de la silhouette para cada número de clúster

print("Scores de la silhouette para cada número de clúster:")
for n_clusters, score in zip(range(min_clusters, max_clusters + 1), scores):
    print(f"Número de clústeres: {n_clusters}, Score de la silhouette: {score:.3f}")

from sklearn.metrics import silhouette_score

# Calculem el coeficient de silhouette
silhouette_avg_df_pca = silhouette_score(df_pca, hierarchical_labels)

print('\n'f"El coeficiente de silhouette promedio es: {silhouette_avg_df_pca}")

Scores de la silhouette para cada número de clúster:
Número de clústeres: 2, Score de la silhouette: 0.889
Número de clústeres: 3, Score de la silhouette: 0.877
Número de clústeres: 4, Score de la silhouette: 0.803
Número de clústeres: 5, Score de la silhouette: 0.807
Número de clústeres: 6, Score de la silhouette: 0.808
Número de clústeres: 7, Score de la silhouette: 0.809
Número de clústeres: 8, Score de la silhouette: 0.809
Número de clústeres: 9, Score de la silhouette: 0.810
Número de clústeres: 10, Score de la silhouette: 0.810

El coeficiente de silhouette promedio es: 0.809807545024122

	status_id	status_type	status_published	num_reactions	num_comments	num_shares	num_likes	num_loves	num_wows	num_hahas	num_sads	num_angrys	Column1	Column2	Column3	Column4
0	246675545449582_1649696485147474	video	4/22/2018 6:00	529	512	262	432	92	3	1	1	0	NaN	NaN	NaN	NaN
1	246675545449582_1649426988507757	photo	4/21/2018 22:45	150	0	0	150	0	0	0	0	0	NaN	NaN	NaN	NaN
2	246675545449582_1648730588577397	video	4/21/2018 6:17	227	236	57	204	21	1	1	0	0	NaN	NaN	NaN	NaN
3	246675545449582_1648576705259452	photo	4/21/2018 2:29	111	0	0	111	0	0	0	0	0	NaN	NaN	NaN	NaN
4	246675545449582_1645700502213739	photo	4/18/2018 3:22	213	0	0	204	9	0	0	0	0	NaN	NaN	NaN	NaN
...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...
7045	1050855161656896_1061863470556065	photo	9/24/2016 2:58	89	0	0	89	0	0	0	0	0	NaN	NaN	NaN	NaN
7046	1050855161656896_1061334757275603	photo	9/23/2016 11:19	16	0	0	14	1	0	1	0	0	NaN	NaN	NaN	NaN
7047	1050855161656896_1060126464063099	photo	9/21/2016 23:03	2	0	0	1	1	0	0	0	0	NaN	NaN	NaN	NaN
7048	1050855161656896_1058663487542730	photo	9/20/2016 0:43	351	12	22	349	2	0	0	0	0	NaN	NaN	NaN	NaN
7049	1050855161656896_1050858841656528	photo	9/10/2016 10:30	17	0	0	17	0	0	0	0	0	NaN	NaN	NaN	NaN

	num_reactions	num_comments	num_shares	num_likes	num_loves	num_wows	num_hahas	num_sads	num_angrys	Column1	Column2	Column3	Column4
count	7050.0	7050.0	7050.0	7050.0	7050.0	7050.0	7050.0	7050.0	7050.0	0.0	0.0	0.0	0.0
mean	230.0	224.0	40.0	215.0	13.0	1.0	1.0	0.0	0.0	NaN	NaN	NaN	NaN
std	463.0	890.0	132.0	449.0	40.0	9.0	4.0	2.0	1.0	NaN	NaN	NaN	NaN
min	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	NaN	NaN	NaN	NaN
25%	17.0	0.0	0.0	17.0	0.0	0.0	0.0	0.0	0.0	NaN	NaN	NaN	NaN
50%	60.0	4.0	0.0	58.0	0.0	0.0	0.0	0.0	0.0	NaN	NaN	NaN	NaN
75%	219.0	23.0	4.0	185.0	3.0	0.0	0.0	0.0	0.0	NaN	NaN	NaN	NaN
max	4710.0	20990.0	3424.0	4710.0	657.0	278.0	157.0	51.0	31.0	NaN	NaN	NaN	NaN

	count	mean	std	min	25%	50%	75%	max
num_reactions	7050.0	230.0	463.0	0.0	17.0	60.0	219.0	4710.0
num_comments	7050.0	224.0	890.0	0.0	0.0	4.0	23.0	20990.0
num_shares	7050.0	40.0	132.0	0.0	0.0	0.0	4.0	3424.0
num_likes	7050.0	215.0	449.0	0.0	17.0	58.0	185.0	4710.0
num_loves	7050.0	13.0	40.0	0.0	0.0	0.0	3.0	657.0
num_wows	7050.0	1.0	9.0	0.0	0.0	0.0	0.0	278.0
num_hahas	7050.0	1.0	4.0	0.0	0.0	0.0	0.0	157.0
num_sads	7050.0	0.0	2.0	0.0	0.0	0.0	0.0	51.0
num_angrys	7050.0	0.0	1.0	0.0	0.0	0.0	0.0	31.0
Column1	0.0	NaN	NaN	NaN	NaN	NaN	NaN	NaN
Column2	0.0	NaN	NaN	NaN	NaN	NaN	NaN	NaN
Column3	0.0	NaN	NaN	NaN	NaN	NaN	NaN	NaN
Column4	0.0	NaN	NaN	NaN	NaN	NaN	NaN	NaN

	status_id	status_type	status_published	num_reactions	num_comments	num_shares	num_likes	num_loves	num_wows	num_hahas	num_sads	num_angrys	Column1	Column2	Column3	Column4
6221	819700534875473_1002372733274918	video	6/10/2018 3:43	376	20	3	354	19	3	0	0	0	NaN	NaN	NaN	NaN
6222	819700534875473_1001982519980606	photo	6/9/2018 22:53	255	7	4	249	6	0	0	0	0	NaN	NaN	NaN	NaN
6223	819700534875473_1000607730118085	photo	6/7/2018 7:01	1704	21	3	1685	15	2	2	0	0	NaN	NaN	NaN	NaN
6224	819700534875473_999880033524188	status	6/6/2018 2:20	205	39	4	197	4	4	0	0	0	NaN	NaN	NaN	NaN
6225	819700534875473_998824716963053	photo	6/4/2018 6:57	336	42	2	285	5	0	0	46	0	NaN	NaN	NaN	NaN
6226	819700534875473_997075427137982	status	6/1/2018 0:35	111	2	1	110	1	0	0	0	0	NaN	NaN	NaN	NaN
6227	819700534875473_995933077252217	photo	5/30/2018 6:33	357	18	9	352	3	1	1	0	0	NaN	NaN	NaN	NaN
6228	819700534875473_995839410594917	photo	5/30/2018 2:39	114	7	0	109	3	2	0	0	0	NaN	NaN	NaN	NaN
6229	819700534875473_995218893990302	status	5/29/2018 4:39	133	7	1	127	6	0	0	0	0	NaN	NaN	NaN	NaN
6230	819700534875473_993975437447981	photo	5/27/2018 1:31	532	32	4	480	47	5	0	0	0	NaN	NaN	NaN	NaN
6231	819700534875473_993602007485324	status	5/26/2018 8:08	231	16	2	216	3	0	0	12	0	NaN	NaN	NaN	NaN
6232	819700534875473_993049450873913	photo	5/25/2018 7:56	684	38	14	676	7	0	1	0	0	NaN	NaN	NaN	NaN
6233	819700534875473_991757311003127	photo	5/23/2018 1:10	1627	30	3	1607	17	3	0	0	0	NaN	NaN	NaN	NaN
6234	819700534875473_990066331172225	photo	5/20/2018 6:18	791	34	2	770	16	5	0	0	0	NaN	NaN	NaN	NaN
6235	819700534875473_988583821320476	photo	5/17/2018 20:00	1404	39	5	1390	12	1	1	0	0	NaN	NaN	NaN	NaN
6236	819700534875473_985229911655867	photo	5/12/2018 1:57	824	106	6	810	10	4	0	0	0	NaN	NaN	NaN	NaN
6237	819700534875473_984750728370452	status	5/11/2018 3:23	295	18	0	288	4	0	0	3	0	NaN	NaN	NaN	NaN
6238	819700534875473_983794271799431	photo	5/9/2018 8:45	1957	124	10	1908	27	21	1	0	0	NaN	NaN	NaN	NaN
6239	819700534875473_981835315328660	photo	5/5/2018 11:20	1730	161	4	1710	12	8	0	0	0	NaN	NaN	NaN	NaN
6240	819700534875473_980941638751361	photo	5/3/2018 23:57	1655	47	4	1632	9	13	1	0	0	NaN	NaN	NaN	NaN
6241	819700534875473_978746665637525	status	4/30/2018 6:20	237	67	1	233	3	1	0	0	0	NaN	NaN	NaN	NaN
6242	819700534875473_977201195792072	photo	4/27/2018 23:27	360	22	10	347	2	5	0	6	0	NaN	NaN	NaN	NaN
6243	819700534875473_976401089205416	photo	4/26/2018 19:00	2635	308	11	2605	19	11	0	0	0	NaN	NaN	NaN	NaN
6244	819700534875473_974461939399331	photo	4/23/2018 7:26	324	45	3	280	0	17	1	19	7	NaN	NaN	NaN	NaN
6245	819700534875473_972433889602136	photo	4/20/2018 3:16	1271	94	9	1249	11	8	2	1	0	NaN	NaN	NaN	NaN
6246	819700534875473_971940472984811	photo	4/19/2018 7:05	851	100	11	834	5	10	2	0	0	NaN	NaN	NaN	NaN
6247	819700534875473_968264653352393	photo	4/12/2018 3:42	1992	36	1	1969	16	7	0	0	0	NaN	NaN	NaN	NaN
6248	819700534875473_967787160066809	photo	4/11/2018 3:43	4315	24	3	4241	51	19	4	0	0	NaN	NaN	NaN	NaN
6249	819700534875473_966878030157722	photo	4/9/2018 10:49	4114	104	3	4054	36	23	0	1	0	NaN	NaN	NaN	NaN
6250	819700534875473_966629653515893	photo	4/8/2018 22:50	95	3	2	90	2	3	0	0	0	NaN	NaN	NaN	NaN
6251	819700534875473_966156923563166	photo	4/8/2018 1:08	111	9	2	107	1	2	1	0	0	NaN	NaN	NaN	NaN
6252	819700534875473_965614580284067	photo	4/7/2018 3:23	796	143	9	791	5	0	0	0	0	NaN	NaN	NaN	NaN
6253	819700534875473_965407870304738	photo	4/6/2018 16:34	508	12	3	499	8	0	1	0	0	NaN	NaN	NaN	NaN
6254	819700534875473_964582740387251	photo	4/5/2018 10:47	2506	287	28	2483	9	12	2	0	0	NaN	NaN	NaN	NaN
6255	819700534875473_963754250470100	photo	4/4/2018 2:20	251	29	5	228	0	3	0	1	19	NaN	NaN	NaN	NaN
6256	819700534875473_962799373898921	photo	4/2/2018 10:16	1244	218	9	1226	5	12	1	0	0	NaN	NaN	NaN	NaN
6257	819700534875473_962694150576110	photo	4/2/2018 7:33	75	2	0	71	3	0	1	0	0	NaN	NaN	NaN	NaN
6258	819700534875473_962061653972693	photo	4/1/2018 5:47	867	90	11	858	1	7	0	0	1	NaN	NaN	NaN	NaN
6259	819700534875473_962032993975559	photo	4/1/2018 4:32	87	5	2	79	3	5	0	0	0	NaN	NaN	NaN	NaN
6260	819700534875473_960466324132226	photo	3/29/2018 13:54	614	78	15	599	7	8	0	0	0	NaN	NaN	NaN	NaN
6261	819700534875473_960320564146802	photo	3/29/2018 8:23	2420	75	17	2392	20	8	0	0	0	NaN	NaN	NaN	NaN
6262	819700534875473_958291217683070	photo	3/26/2018 1:57	86	5	3	80	2	4	0	0	0	NaN	NaN	NaN	NaN
6263	819700534875473_957697307742461	photo	3/25/2018 4:43	488	20	8	484	4	0	0	0	0	NaN	NaN	NaN	NaN
6264	819700534875473_957599447752247	photo	3/24/2018 23:51	157	16	3	145	1	11	0	0	0	NaN	NaN	NaN	NaN
6265	819700534875473_956769714501887	photo	3/23/2018 15:53	547	71	7	535	6	6	0	0	0	NaN	NaN	NaN	NaN
6266	819700534875473_955793427932849	photo	3/22/2018 6:10	1168	104	49	1133	17	17	1	0	0	NaN	NaN	NaN	NaN
6267	819700534875473_955743124604546	photo	3/22/2018 3:48	879	165	18	867	4	8	0	0	0	NaN	NaN	NaN	NaN
6268	819700534875473_955149101330615	photo	3/21/2018 1:15	114	6	1	108	3	3	0	0	0	NaN	NaN	NaN	NaN
6269	819700534875473_954387151406810	photo	3/20/2018 1:54	186	15	1	172	3	11	0	0	0	NaN	NaN	NaN	NaN
6270	819700534875473_953048221540703	photo	3/18/2018 4:28	1985	39	21	1961	11	12	0	1	0	NaN	NaN	NaN	NaN
6271	819700534875473_951614605017398	photo	3/16/2018 1:57	985	71	42	962	5	16	2	0	0	NaN	NaN	NaN	NaN

	pca1	pca2
0	48.144989	39.282941
1	-14.871880	-2.243211
2	3.374302	3.442291
3	-14.874602	-2.241792
4	-14.264812	-1.388950
...	...	...
6994	-14.880681	-2.250199
6995	-14.789026	-2.144672
6996	-14.824033	-2.179699
6997	-11.064553	1.579700
6998	-14.888441	-2.265234

Cristina Cosma¶

DATA SCIENCE¶

Sprint 8. ML - Algoritmos de aprendizage no supervisado (agrupación)¶

0. Importar las librerías necesarias, cargar el dataset, analizar preparar los datos¶

Transformaciones báscas y anális de variables¶

Preparar el dataset : limpieza e interpretación de los datos¶

Para objetivos del análisis estadístico, se procede a verificar las variables relacionadas con el usuario y el estátus.¶

Vamos a separar el status_published para obtener datos temporales y borrar algunas columnas que repiten datos ya transformados.¶

Anális descriptivo de las variables más interesantes¶

Limpieza del dataset para proceder a los algorítmos no supervisados¶

Observaciones sobre el gráfico de correlaciones :¶

Evaluación de distribuciones / Tests de normalidad¶

Busca de valores atípicos / outliers¶

Resumen de las columnas :¶

ESCALADO:¶

Exercici 1: Clasificar los diferentes registros utilizando el algoritmo K-means¶

Veamos lo que contiene cada uno de los 5 clústers generados por k-means¶

Organzación detallada de los 5 clústers aglomerados por k-means¶

Observaciones sobre las gráficas de los 5 clústers resultantes tras entrenar el modelo k-means:¶

PCA¶

Ejercicio 2: Clasificar los diferentes registros utilizando el algoritmo de clustering jerárquico¶

Ejercicio 3: Calcula el rendiment del clustering mitjançant un paràmetre com pot ser silhouette¶

Calculado los Silhouette scores para el mismo número de clústeres => n_clusters=5¶

Calculado los Silhouette scores para diferentes números de clústeres:¶

En conclusión¶

	status_id	status_type	status_published	num_reactions	num_comments	num_shares	num_likes	num_loves	num_wows	num_hahas	...	year	status_type_photo	status_type_video	time_day_morning	time_day_night	month	minute
0	246675545449582_1649696485147474	video	2018-04-22 06:00:00	529	512	262	432	92	3	1	...	2018	0	1	1	0	4	0
1	246675545449582_1649426988507757	photo	2018-04-21 22:45:00	150	0	0	150	0	0	0	...	2018	1	0	0	1	4	45
2	246675545449582_1648730588577397	video	2018-04-21 06:17:00	227	236	57	204	21	1	1	...	2018	0	1	1	0	4	17
3	246675545449582_1648576705259452	photo	2018-04-21 02:29:00	111	0	0	111	0	0	0	...	2018	1	0	0	1	4	29
4	246675545449582_1645700502213739	photo	2018-04-18 03:22:00	213	0	0	204	9	0	0	...	2018	1	0	0	1	4	22

	count	mean	std	min	25%	50%	75%	max
num_reactions	6999.0	225.0	453.0	0.0	17.0	58.0	216.0	4710.0
num_comments	6999.0	226.0	893.0	0.0	0.0	4.0	22.0	20990.0
num_shares	6999.0	40.0	132.0	0.0	0.0	0.0	4.0	3424.0
num_likes	6999.0	210.0	440.0	0.0	17.0	57.0	182.0	4710.0
num_loves	6999.0	13.0	40.0	0.0	0.0	0.0	3.0	657.0
num_wows	6999.0	1.0	9.0	0.0	0.0	0.0	0.0	278.0
num_hahas	6999.0	1.0	4.0	0.0	0.0	0.0	0.0	157.0
num_sads	6999.0	0.0	1.0	0.0	0.0	0.0	0.0	51.0
num_angrys	6999.0	0.0	1.0	0.0	0.0	0.0	0.0	31.0
day	6999.0	16.0	9.0	1.0	8.0	16.0	23.0	31.0
hour	6999.0	8.0	7.0	0.0	2.0	7.0	9.0	23.0
Saturday	6999.0	0.0	0.0	0.0	0.0	0.0	0.0	1.0
Sunday	6999.0	0.0	0.0	0.0	0.0	0.0	0.0	1.0
Workday	6999.0	1.0	0.0	0.0	0.0	1.0	1.0	1.0
year	6999.0	2016.0	2.0	2012.0	2016.0	2017.0	2018.0	2018.0
status_type_link	6999.0	0.0	0.0	0.0	0.0	0.0	0.0	1.0
status_type_photo	6999.0	1.0	0.0	0.0	0.0	1.0	1.0	1.0
status_type_status	6999.0	0.0	0.0	0.0	0.0	0.0	0.0	1.0
status_type_video	6999.0	0.0	0.0	0.0	0.0	0.0	1.0	1.0
time_day_afternoon/evening	6999.0	0.0	0.0	0.0	0.0	0.0	0.0	1.0
time_day_morning	6999.0	0.0	0.0	0.0	0.0	0.0	1.0	1.0
time_day_night	6999.0	1.0	0.0	0.0	0.0	1.0	1.0	1.0
month	6999.0	7.0	4.0	1.0	3.0	6.0	10.0	12.0
minute	6999.0	27.0	18.0	0.0	12.0	26.0	43.0	59.0
ID_1050855161656896	6999.0	0.0	0.0	0.0	0.0	0.0	0.0	1.0
ID_117059915488335	6999.0	0.0	0.0	0.0	0.0	0.0	0.0	1.0
ID_134115277150304	6999.0	0.0	0.0	0.0	0.0	0.0	0.0	1.0
ID_146481362756646	6999.0	0.0	0.0	0.0	0.0	0.0	0.0	1.0
ID_246675545449582	6999.0	0.0	0.0	0.0	0.0	0.0	1.0	1.0
ID_614855718638584	6999.0	0.0	0.0	0.0	0.0	0.0	0.0	1.0
ID_725980084255106	6999.0	0.0	0.0	0.0	0.0	0.0	0.0	1.0
ID_819700534875473	6999.0	0.0	0.0	0.0	0.0	0.0	0.0	1.0
ID_848451358648417	6999.0	0.0	0.0	0.0	0.0	0.0	0.0	1.0

	num_reactions	num_comments	num_shares	num_likes	num_loves	num_wows	num_hahas	num_sads	day	...	minute	ID_246675545449582
0	2.366834	23.090909	65.50	2.272727	30.666667	3.0	1.0	1.0	0.700000	...	0.000000	1
1	0.462312	-0.181818	0.00	0.563636	0.000000	0.0	0.0	0.0	0.666667	...	0.762712	1
2	0.849246	10.545455	14.25	0.890909	7.000000	1.0	1.0	0.0	0.666667	...	0.288136	1
3	0.266332	-0.181818	0.00	0.327273	0.000000	0.0	0.0	0.0	0.666667	...	0.491525	1
4	0.778894	-0.181818	0.00	0.890909	3.000000	0.0	0.0	0.0	0.566667	...	0.372881	1